# Scrapy settings for itemextract project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'itemextract'

SPIDER_MODULES = ['itemextract.spiders']
NEWSPIDER_MODULE = 'itemextract.spiders'
DOWNLOAD_HANDLERS = {
    'http': 'itemextract.dhandler.WebkitDownloadHandler',
    'https': 'itemextract.dhandler.WebkitDownloadHandler',
}
ITEM_PIPELINES = {
        'itemextract.pipelines.ItemextractPipeline':300,
    }
SPIDER_MIDDLEWARES={
        'itemextract.spidermiddlewares.CustomSpiderMiddleware': 543,        
    }
DOWNLOADER_MIDDLEWARES={
        'itemextract.downloadermiddlewares.CustomDownloaderMiddle':543,        
    }
EXTENSIONS={
            'itemextract.extensions.ExtTest':500,
            }
MYEXT_ENABLED = True
PAGES_PER_PATH=60
#CONCURRENT_REQUESTS = 1
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'itemextract (+http://www.yourdomain.com)'
